#!/usr/bin/env python
# encoding: utf-8
"""
@author: youfeng
@email: youfeng243@163.com
@license: Apache Licence 
@file: start_parse_db.py
@time: 2016/12/22 15:37
"""
import time
from multiprocessing import Pool

from common import util
from common.config_parser import ConfigParser
from logger import Gsxtlogger
from parse.generator import create_object

log = Gsxtlogger(util.get_pid_log_name('start_parse_db')).get_logger()


# data = {_id: xxx, in_time: xxx, company_name:xxx, province: xxx, base_info:{} }
# target_table: cmb_base_info_anhui

def run_task(config):
    province = config.get('province', '')
    if province != 'sichuan' and province != 'xinjiang':
        return

    worker = create_object(config)

    start_time = time.time()
    log.info('开始清洗数据.....')
    worker.start_task()
    end_time = time.time()
    log.info('数据清洗完成.....')
    log.info('start_time: {start}'.format(start=start_time))
    log.info('end_time: {end}'.format(end=end_time))
    log.info('used = {used}'.format(used=(end_time - start_time)))
    log.info('')


def main():
    config_file = 'config/cmb_gsxt_search.conf'
    conf_parse = ConfigParser(config_file)
    config_list = conf_parse.get_all_session()
    for key, value in config_list.iteritems():
        source_table = value.get('target_table', '')
        value['source_table'] = source_table
        value['target_table'] = 'cmb_base_info_' + value.get('province', '')

    pool = Pool(processes=10)

    pool.map(run_task, config_list.itervalues())

    pool.close()
    pool.join()


if __name__ == '__main__':
    main()
